/*
 * Copyright (c) 2008-2009, Computational Crawling LP
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 *    * Neither the name of Computational Crawling LP, 80legs, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.eightylegs.customer.default80legs;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Class KeywordRegexAnalysis.
 * 
 * This class implements the content selection that is a default option available through the 80legs portal. 
 */
public class KeywordRegexAnalysis {
	private static int maxTimePerPageMS = 5000;
	private static int stringFreqCheckTime = 1;
	
	/**
	 * process a single page using the CrawlSelectData specified by the user input
	 * the result format is depended on the returnType specified by the user input
	 * 
	 * @param urlName  Name of the page being processed
	 * @param pageContents  The contents of the page as a byte[]
	 * @param data  The CrawlSelectData object representing the user's parameters, strings, and regexes
	 * 
	 * @return   The results of the processPage function serialized as a byte[]
	 */
	static public byte[] processPage(String urlName, byte[] pageContents, CrawlSelectData data) {
		try {
			String textContents = ClientUtility.byteArrayToString( pageContents, 0, pageContents.length);
			return processPage(urlName, textContents, data) ;
		} catch (Exception e) {
			return null;
		}
	}
	
	/**
	 * process a single page using the CrawlSelectData specified by the user input
	 * the result format is depended on the returnType specified by the user input
	 * 
	 * @param urlName  Name of the page being processed
	 * @param inTextContents  The contents of the page as a String
	 * @param data  The CrawlSelectData object representing the user's parameters, strings, and regexes
	 * 
	 * @return   The results of the processPage function serialized as a byte[]
	 */
	static public byte[] processPage(String urlName, String inTextContents, CrawlSelectData data) {
		try {
			String textContents = inTextContents.toLowerCase();

			byte[] tmpResult = null;

			int returnType = data.GetReturnType();
			ArrayList<ContentSelectUserItem> userItems = data.GetItems();

			if (returnType == CrawlSelectData.CSD_COUNTMATCHES) {
				tmpResult = processPageCountMatches(textContents, userItems);
			}
			else if (returnType == CrawlSelectData.CSD_BOOLEANARRAY) {
				tmpResult = processPageBooleanArray(textContents, userItems);
			}
			else if (returnType == CrawlSelectData.CSD_COUNTARRAY) {
				tmpResult = processPageCountArray(textContents, userItems);
			}

			return tmpResult;
		} catch (Exception e) {
			return null;
		}
	}

	/**
	 *  Computes two different counts based on the pageContents and the user's items:
	 *   * uniqueCount = the total number of unique matches of items on the page
	 *   * totalCount = the total number of matches of items on the page
	 *   
	 *  Example:  The contents of the first page are 'test1 test1 test2 test3 test5'. The contents of the second page are 'this is a test'.
	 *     The keyword list is:
	 *       test
	 *       test1
	 *       test2
	 *       test3
	 *       test4
	 *       test5
	 *       test6
	 *       
	 *     For page1, the uniqueCount is 5 and totalCount is 10
	 *     For page2, the uniqueCount is 1 and totalCount is 1
	 * 
	 * @param pageContents  The contents of the page as a String
	 * @param items  The user's items
	 * 
	 * @return   The results of the processPage function serialized as a byte[]
	 */
	static private byte[] processPageCountMatches(String pageContents, ArrayList<ContentSelectUserItem> items) throws Exception {
		long startTimeMS = System.currentTimeMillis();

		int totalCount = 0;
		int uniqueCount = 0;
		int iloopCount = 0;
		for ( ContentSelectUserItem curItem : items ) {
			if ((iloopCount % stringFreqCheckTime) == 0) {
				long curTimeMS = System.currentTimeMillis();
				if (curTimeMS - startTimeMS > maxTimePerPageMS) {
					ClientUtility.outMessage("Error1");
					return null; // subwork took too long
				}
			}
			iloopCount++;
			
			if ( curItem.IsRegex() == true ) {
				Pattern curPattern = curItem.GetPattern();
				if ( curPattern != null )
				{
					Matcher curMatcher = curPattern.matcher(pageContents);
					int itemCount = 0;
					while ( curMatcher.find() == true )
						itemCount++;
					if ( itemCount > 0 )  {
						uniqueCount++;
						totalCount += itemCount;
					}
						
				}
			} else {
				// count the number of times the user string is in the
				// pageContents
				int itemCount = 0;
				int startIndex = 0;
				String itemString = curItem.GetString();
				while ((startIndex = pageContents.indexOf(itemString, startIndex)) != -1) {
					itemCount++;
					startIndex++;
					//System.out.println ( "Found " + itemString );
				}
				if ( itemCount > 0 )
				{
					uniqueCount++;
					totalCount += itemCount;
				}
			}
		}

		byte[] result = new byte[12];
		ClientUtility.intToByteArray(totalCount, result, 0);
		ClientUtility.intToByteArray(uniqueCount, result, 4);
		ClientUtility.intToByteArray(items.size(), result, 8);
		return result;
	}

	/**
	 *  Computes whether each item in items exists on the page or not
	 *   
	 * @param pageContents  The contents of the page as a String
	 * @param items  The user's items
	 * 
	 * @return   The results of the processPage function serialized as a byte[]
	 */
	static private byte[] processPageBooleanArray(String pageContents, ArrayList<ContentSelectUserItem> items) throws Exception {
		long startTimeMS = System.currentTimeMillis();

		int maxBytes = items.size() / 8;
		if ((items.size() % 8) > 0)  {
			maxBytes++;
		}

		// put the counts in the results
		byte[] countBytes = processPageCountMatches( pageContents, items );
		byte[] results = new byte[countBytes.length+maxBytes];
		System.arraycopy(countBytes, 0, results, 0, countBytes.length);
		int byteStart = countBytes.length;
		maxBytes += byteStart;
		
		int whichByte = byteStart;
		int whichBit = 0;
		results[byteStart] = 0;
		int iloopCount = 0;
		for ( ContentSelectUserItem curItem : items ) {
			if ((iloopCount % stringFreqCheckTime) == 0) {
				long curTimeMS = System.currentTimeMillis();
				if (curTimeMS - startTimeMS > maxTimePerPageMS) {
					ClientUtility.outMessage("Error1");
					return null; // subwork took too long
				}
			}
			iloopCount++;

			boolean goodResult = false;
			if (curItem.IsRegex() == true) {
				Pattern curPattern = curItem.GetPattern();
				if ( curPattern != null ) {
					Matcher curMatcher = curPattern.matcher(pageContents);
					goodResult = curMatcher.find();
				}
			}
			else {
				goodResult = pageContents.contains(curItem.GetString());
			}

			if (goodResult == true) {
				if (whichByte < maxBytes) {
					results[whichByte] += (1 << whichBit);
				}
			}

			whichBit++;
			if (whichBit == 8) {
				whichBit = 0;
				whichByte++;
				if (whichByte < maxBytes) {
					results[whichByte] = 0;
				}
			}
		}

		return results;
	}

	/**
	 *  Computes a 16-bit integer count for each item in items based on the number of times the item matches the page contents
	 *   
	 * @param pageContents  The contents of the page as a String
	 * @param items  The user's items
	 * 
	 * @return   The results of the processPage function serialized as a byte[]
	 */
	static private byte[] processPageCountArray(String pageContents, ArrayList<ContentSelectUserItem> items) throws Exception {
		long startTimeMS = System.currentTimeMillis();

		// put the counts in the results
		byte[] countBytes = processPageCountMatches( pageContents, items );
		byte[] results = new byte[items.size() * 2 + countBytes.length];
		System.arraycopy(countBytes, 0, results, 0, countBytes.length);
		int startByte = countBytes.length;

		int iloopCount = 0;
		for ( ContentSelectUserItem curItem : items ) {
			if ((iloopCount % stringFreqCheckTime) == 0) {
				long curTimeMS = System.currentTimeMillis();
				if (curTimeMS - startTimeMS > maxTimePerPageMS) {
					ClientUtility.outMessage("Error1");
					return null; // subwork took too long
				}
			}

			int itemCount = 0;
			if (curItem.IsRegex() == true) {
				Pattern curPattern = curItem.GetPattern();
				if ( curPattern != null ) {
					Matcher curMatcher = curPattern.matcher(pageContents);
					while ( curMatcher.find() == true )
						itemCount++;
				}
			} else {
				// count the number of times the user string is in the pageContents
				int startIndex = 0;
				String itemString = curItem.GetString();
				while ((startIndex = pageContents.indexOf(itemString, startIndex)) != -1) {
					itemCount++;
					startIndex++;
					//System.out.println("Found " + curItem.GetString());
				}
			}

			if (itemCount > 0) {
				short scount;
				if ( itemCount >= Short.MAX_VALUE )
					scount = Short.MAX_VALUE;
				else
					scount = (short)itemCount;
				ClientUtility.shortToByteArray(scount, results, iloopCount * 2 + startByte);
			}
			iloopCount++;
		}

		return results;
	}
}
